# -*- coding: utf-8 -*-
# @Time    : 2018/12/11 10:47
import os
import sys

path = os.path.abspath(os.path.dirname(os.getcwd()))
sys.path.append(path)
import pymongo,config
from collections import OrderedDict
from multiprocessing import Process, Queue, Pool

db = pymongo.MongoClient("mongodb://192.168.2.211:27017")
li_two = db["Two_hundred_million"]
li_db_words = li_two["new_biying"]
new_url_biying = config.new_li_db_biying

class FindSource(object):

    def __init__(self) -> None:
        self.B = 0
        super().__init__()

    def delete_mongo(self):
        while True:
            try:
                # 关键词
                item = li_db_words.find_and_modify({'state': 0}, {'$set': {"state": 1}})
                if not item:
                    return
                # 插入库里
                self.find_one(item)
            except Exception as e:
                print(e)
    def find_one(self,item):
        save = OrderedDict()
        save['url'] = item['url']
        try:
            save['title'] = item['title']
        except:
            pass
        try:
            save['summary'] = item['summary']
        except:
            pass
        save['state'] = 0  # 状态
        save['url_md5'] = item['url_md5']
        save['type'] = item['type']  # 类型
        save['q'] = item['q']
        try:
            new_url_biying.insert(save)
            self.B += 1
            if self.B % 100 == 0:
                print("charu : %s"%(item['url_md5']))
        except Exception as e:
            if str(e).find("E11000") > -1:
                return
            print("插入失败：%s" % (e))

    def proce(self):
        trader = []
        for i in range(20):
            pr = Process(target=self.delete_mongo)
            pr.start()
            trader.append(pr)
        for i in trader:
            i.join()
        print('proce this is pid: %s' % os.getpid())


if __name__ == '__main__':
    # lin_sql.creat_fulltext()
    FindSource().proce()
    # FindSource().restart_program()
